"""
Case Type   : 操作系统类-文件系统故障
Case Name   : 减容/扩容时，注入故障文件系统inode资源耗尽
Create At   : 2021/8/3
Owner       : li-xin12345
Description :
    1、注入文件系统inode资源耗尽故障
    2、启线程执行减容
    3、清理故障，再次执行减容
    4、注入文件系统inode资源耗尽故障
    5、启线程执行扩容
    6、清理故障，再次执行扩容
    7、运行TPCC
    8、检查数据一致性
    9、恢复环境
Expect      :
    1、故障注入成功
    2、gs_probackup报错“No space left on device”
    3、清理故障成功，减容成功
    4、故障注入成功
    5、扩容报错“No space left on device”
    6、清理故障成功，扩容成功
    7、运行TPCC成功
    8、数据一致
    9、恢复环境
History     :
"""
import os
import time
import unittest

from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import CommonSH
from testcase.utils.CommonSH import RestartDbCluster
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node, macro


class ReliabilityFileSystem(unittest.TestCase):
    nodes = ("PrimaryDbUser", "Standby1DbUser", "Standby2DbUser")

    @RestartDbCluster(*nodes)
    def setUp(self):
        self.log = Logger()
        self.primary_node = Node(self.nodes[0])
        self.s_node1 = Node(self.nodes[1])
        self.primary_sh = CommonSH(self.nodes[0])
        self.s_com1 = CommonSH(self.nodes[1])
        self.s_com2 = CommonSH(self.nodes[2])
        self.primary_root_node = Node(node="PrimaryRoot")
        self.sta1_root_node = Node(node="Standby1Root")
        self.primary_root_sh = CommonSH("PrimaryRoot")
        self.constant = Constant()
        self.com = Common()
        self.file = os.path.join(macro.DB_INSTANCE_PATH,
                                 macro.DB_PG_CONFIG_NAME)
        self.log.info("setUp-实例化:ok")

        self.log.info("Opengauss_Reliability_OS_File_System_Case057:开始执行")
        self.file_name = macro.DB_INSTANCE_PATH
        self.inject_cmd = f"rfile_ideplenish_workmode " \
                          f"(filename, workmode) " \
                          f"values ({self.file_name}, fore)"

        self.clean_cmd = f"rfile_ideplenish_workmode where " \
                         f"filename={self.file_name} and workmode=fore"

    def test_main(self):
        text = "备份postgresql.conf文件"
        self.log.info(text)
        result = self.primary_node.sh(f"cp {self.file} "
                                      f"{self.file}backup").result()
        self.log.info(result)

        try:
            text = "--step1:待减容节点注入文件系统inode资源耗尽故障;expect:成功"
            self.log.info(text)
            self.log.info("注入故障")
            session1 = ComThread(self.com.cfe_inject,
                                 args=(self.sta1_root_node,
                                       self.inject_cmd))
            session1.setDaemon(True)
            session1.start()
            time.sleep(1)

            self.log.info("故障注入结果")
            session1.join(60 * 30)
            result = session1.get_result()
            self.log.info(result)
            self.assertIn("successful execution", result, "执行失败" + text)

            text = "--step2：执行减容"
            self.log.info(text)
            self.log.info("启线程执行减容")
            execute_cmd = f'''source {macro.DB_ENV_PATH};
                        expect <<EOF
                        set timeout 120
                        spawn gs_dropnode -U {self.primary_node.ssh_user} \
                        -G {self.primary_node.ssh_user} \
                        -h {self.s_node1.ssh_host}
                        expect "*drop the target node (yes/no)?*"
                        send "yes\\n"
                        expect eof\n''' + '''EOF'''
            self.log.info(execute_cmd)
            session2 = ComThread(self.com.get_sh_result,
                                 args=(self.primary_node, execute_cmd))
            session2.setDaemon(True)
            session2.start()
            time.sleep(2)

            self.log.info("减容结果")
            session2.join(60 * 30)
            result = session2.get_result()
            self.log.info(result)
            self.assertIn("Success to drop the target nodes", result,
                          "执行失败" + text)
            self.assertNotIn(self.constant.DISK_FULL_MSG, result,
                             "执行失败" + text)

        finally:
            text = "--step3:清除故障;expect:成功"
            self.log.info(text)
            result = self.com.cfe_clean(self.sta1_root_node,
                                        self.clean_cmd)
            self.log.info(result)
            self.assertIn("successful execution", result, "执行失败" + text)

            text = "--step4:再次减容"
            self.log.info(text)
            execute_cmd = f'''source {macro.DB_ENV_PATH};
                        expect <<EOF
                        set timeout 120
                        spawn gs_dropnode -U {self.primary_node.ssh_user} \
                        -G {self.primary_node.ssh_user} \
                        -h {self.s_node1.ssh_host}
                        expect "*drop the target node (yes/no)?*"
                        send "yes\\n"
                        expect eof\n''' + '''EOF'''
            self.log.info(execute_cmd)
            result = self.primary_node.sh(execute_cmd).result()
            self.log.info(result)
            self.assertIn("Success to drop the target nodes", result,
                          "执行失败" + text)
            self.assertNotIn(self.constant.DISK_FULL_MSG, result,
                             "执行失败" + text)

        self.log.info("减容测试结束 开始扩容测试")

        try:
            text = "--step5:待减容节点注入inode资源耗尽故障;expect:成功"
            self.log.info(text)
            self.log.info("注入故障")
            session1 = ComThread(self.com.cfe_inject,
                                 args=(self.sta1_root_node,
                                       self.inject_cmd))
            session1.setDaemon(True)
            session1.start()
            time.sleep(1)

            self.log.info("故障注入结果")
            session1.join(60 * 30)
            result = session1.get_result()
            self.log.info(result)
            self.assertIn("successful execution", result, "执行失败" + text)

            text = "--step5：执行扩容"
            self.log.info(text)
            self.log.info("启线程扩容")
            session2 = ComThread(self.primary_root_sh.exec_expension,
                                 args=(f"{self.primary_node.ssh_user}",
                                       f"{self.primary_node.ssh_user}",
                                       f"{self.s_node1.ssh_host}",
                                       f"{macro.DB_XML_PATH}"))
            session2.setDaemon(True)
            session2.start()
            time.sleep(2)

            self.log.info("扩容结果")
            session2.join(60 * 30)
            result = session2.get_result()
            self.log.info(result)
            self.assertFalse(result, "执行失败" + text)

        finally:
            text = "--step6:清除故障;expect:成功"
            self.log.info(text)
            result = self.com.cfe_clean(self.sta1_root_node,
                                        self.clean_cmd)
            self.log.info(result)
            self.assertIn("successful execution", result, "执行失败" + text)

            text = "--step6:再次扩容"
            self.log.info(text)
            result = self.primary_root_sh.exec_expension(
                f"{self.primary_node.ssh_user}",
                f"{self.primary_node.ssh_user}",
                f"{self.s_node1.ssh_host}",
                f"{macro.DB_XML_PATH}",
                detail=True)
            self.log.info(result)
            self.assertIn("Expansion Finish", result, "执行失败" + text)
            self.assertNotIn(self.constant.DISK_FULL_MSG, result,
                             "执行失败" + text)

        db_status = self.primary_sh.getDbClusterStatus("detail")
        self.log.info(db_status)
        self.assertTrue("Degraded" in db_status or "Normal" in db_status)

        text = "--step7:运行TPCC;expect:成功"
        self.log.info(text)
        result = self.com.startTPCC(self.primary_root_node, macro.TPCC_PATH)
        self.log.info(result)
        self.assertIn(self.constant.TPCC_SUCCESS_MSG, result, "执行失败" + text)

        self.log.info("检查主备是否同步")
        time.sleep(10)
        self.log.info("获取节点数量")
        node_num = self.primary_sh.get_node_num()
        res = self.primary_sh.check_location_consistency("primary", node_num)
        self.assertTrue(res, "执行失败" + text)

        text = "--step8:检查数据一致性;expect:成功"
        self.log.info(text)
        nodes = (Node(self.nodes[0]), Node(self.nodes[1]),
                 Node(self.nodes[2]))
        flag = self.com.check_data_sample_by_all(r'\d', *nodes)
        self.assertTrue(flag, "执行失败" + text)
        table_dict = self.com.format_sql_result(
            self.primary_sh.executDbSql(r'\d'))
        table_name = table_dict.get('Name')
        for name in table_name:
            select_sql = f"select count(*) from {name};"
            flag = self.com.check_data_sample_by_all(select_sql, *nodes)
            self.assertTrue(flag, "执行失败" + text)

    def tearDown(self):
        text = "--step9:环境清理;expect:成功"
        self.log.info(text)
        res = self.com.cfe_clean(self.sta1_root_node,
                                 self.clean_cmd)
        self.log.info(res)

        self.log.info("扩容")
        result = self.primary_sh.executDbSql(
            "select count(*) from pg_stat_get_wal_senders();")
        self.log.info(result)
        if "2" != result.split("\n")[-2].strip():
            result = self.primary_root_sh.exec_expension(
                f"{self.primary_node.ssh_user}",
                f"{self.primary_node.ssh_user}",
                f"{self.s_node1.ssh_host}",
                f"{macro.DB_XML_PATH}",
                detail=True)
            self.log.info(result)
            self.assertIn("Expansion Finish", result, "执行失败" + text)

        self.log.info("恢复postgresql.conf文件")
        mv_result = self.primary_node.sh(f"mv {self.file}backup "
                                         f"{self.file}").result()
        self.log.info(mv_result)

        self.assertIn("successful execution", res)
        self.assertEqual("", mv_result, "执行失败:" + text)
        self.log.info("故障已清理")
        self.log.info("Opengauss_Reliability_OS_File_System_Case057:执行完成")
